#!/usr/bin/env python3

from config import *
import pymongo
from multiprocessing import Pool
import pandas as pd

#配置mongodb的信息
client = pymongo.MongoClient(MONGO_URL)
#连接paper_info库
db_paper_info = client[MONGO_PAPER_INFO_DB]
#连接researcher_info库
db_researcher_sex_info =client[MONGO_RESEARCHER_SEX_DB]


'''
读取university_name中的数据并返回为pandas格式的数据
'''
def get_db_data(university_name):
    university_info_table = db_paper_info[university_name]
    university_data = pd.DataFrame(list(university_info_table.find()))
    return university_data

'''
获取到university_name中的所有含有affiliation的researcher数据,返回格式是list
'''
def get_university_all_info(university_name):
    university_data = get_db_data(university_name)
    list = []
    for data_item in university_data.values:
        # print(data_item)
        for researcher_item in data_item[0]:
            #判断论文中出现的作者是否存在学校或是相应的机构
            researcher_affiliation = researcher_item.get("AfN")
            print(researcher_item)
            if researcher_affiliation:
                #判断接下来要存入list中的人是否存在于list中
                researcher_data = {"AuN":researcher_item.get("AuN"),"AuId":researcher_item.get("AuId"),"AfN":researcher_item.get("AfN"),"Sex":1}
                if researcher_data not in list:
                    list.append(researcher_data)
                    db_researcher_sex_info[university_name].save(researcher_data)

    return list

'''
将所有的学校的researcher信息汇总到一个collection中
'''
def save_all_researcher_data():
    researcher_list = []
    for university_name in UNIVERSITY_LIST:
            # for item in university_data:
        university_data = get_db_data(university_name)
        for researcher_item in university_data.values[0]:
            researcher_id = researcher_item.get("AuId")
            if researcher_id:
                researcher_data = {"AuN":researcher_item.get("AuN"),"AuId":researcher_item.get("AuId"),"AfN":researcher_item.get("AfN"),"Sex":researcher_item.get("Sex")}
                if researcher_data not in researcher_list:
                    researcher_list.append(researcher_data)
                    db_researcher_sex_info["all_researcher_data"].save(researcher_data)

    return len(researcher_list)

def main():
    # demo = db_researcher_sex_info["sex_info"]
    # for item in demo.find():
    #     print(item)
    university_data_list = []
    pool = Pool()
    university_data_list.append(pool.map(get_university_all_info,UNIVERSITY_LIST))
    print(len(university_data_list))

if __name__ == '__main__':
    main()
